package com.laivi.basic.model.crawler;

import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * @author laivi.zhu@gmail.com
 * @date 2013-10-1
 * description:
 */
public class LinkParser {

	public static Set<String> extractLinks(String url,LinkFilter filter){
		Set<String> links = new HashSet<String>();
		try {
			Document doc =Jsoup.connect(url).ignoreContentType(true).cookie("auth","token").header("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2").timeout(1000*50).get();
			Elements link=doc.select("a[href]");
			for(Element e:link){
				String href=e.attr("abs:href").trim();
				if(filter.accept(href)){
					links.add(href);
				}
			}
		} catch (IOException e) {
			e.printStackTrace();
		}catch(Exception e){
			e.printStackTrace();
		}
		return links;
	}
	
	public static Set<String> extractLinks(String url){
		Set<String> links = new HashSet<String>();
		try {
			Document doc =Jsoup.connect(url).ignoreContentType(true).cookie("auth","token").header("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2").timeout(1000*50).get();
			Elements link=doc.select("a[href]");
			for(Element e:link){
				String href=e.attr("abs:href").trim();
				links.add(href);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}catch(Exception e){
			e.printStackTrace();
		}
		return links;
	}
}
